#!/usr/bin/perl
#
use Text::ParseWords;
use URI::Escape;

# skip hosts
@skiphosts = (	'googlebot.com',
		'search.msn.com',
		'localhost.localdomain',
		'pingdom.com$',
		'.crawl.baidu.com$',
		'crawler.sistrix.net',
		'yandex.com',
		'180.76.6.',
		'85.17.56.99',
		'55.138.87.109.triolan.net',
		'yst.yahoo.net$',
		'yandex.ru$',
		'.scoutjet.com$',
		'abcd-ovh4.yasni.de'
	);

@skipbrowsers = (
		'^FeedBurner',
		'^panscient.com',
		'^Sogou web spider',
		'^Links SQL'
	);

@skipurls = (	'^/vufindsmu/favicon.ico',
		'^/vufindsmu/interface/themes',
		'^/vufindsmu/bookcover.php',
		'/favicon.ico$',
		'^/vufindsmu/js',
		'^/vufindsmu/images/',
		'^/vufindsmu/services/Record/ajax.js$',
		'^/robots.txt$',
		'^$',
		'^/[a-u]/',
		'password',
		'^/awstats',
		'ajax.js$',
		'^/vufindsmu/Search/OpenSearch'
	);


my $queryfields1 = { 	 };



%clienttotal;
$skipcount;
$hitcount;
while (<STDIN>) {
	$skip = 0;
	$linecount++;
	$line = $_;
	@fields = quotewords('\s+', 0, $line); # split the line, but keep stuff in quotes as one field
	# 0=host name 3=date/time 5 GET URL and http ver 6= response status 7=size 8=refer 9=browser 
	#print "\n0 " . $fields[0] . "\t3 " . $fields[3] . "\t4 " . $fields[4] . 
	#	"\t5 " . $fields[5]. "\t6 " . $fields[6] . "\t7 " . $fields[7] . "\t8 " .  $fields[8] . "\t9 " .  $fields[9];
	# field 5 contains the GET url - and other stuff  
	@fieldfive =  quotewords('\s+', 0, $fields[5]); 
	$geturl = $fieldfive[1]; # the second subfield (ie 1) of five is the url
	#
	# Skip if we don't want this host, e.g. the hit is from a crawler
	foreach (@skiphosts) {
                if ($fields[0] =~ m/$_/ ) { $skip = 1; }
        }
	foreach (@skipbrowsers) {
		if ($fields[9] =~ m/$_/ ) { $skip = 1; }
	}

	# Skip if we are not interested in the get request (e.g. request for js and css files)
	foreach (@skipurls) {
		if ($geturl =~ m/$_/ ) { $skip = 1;}
	}
	if ($skip) {
                #print "SKIP $fields[0] $geturl \n";
		$skipcount++;
                next;
        }
	if ($fields[6] eq '302') { #don't record lines that are just redirect to another page.
		$referertotal{$field[8]}++;
		next;
	}
	$hitcount++;
	print "$line";
	
	# break up the URL in to the actual file and the Get parameters 
	($filename, $querystring) = split (/\?/, $geturl);
	if ($filename eq "") { $filename = $geturl; }
	@querystrings = split (/&/, $querystring);
	$filetotals{$filename}++;
	# Add each query field/value to a hash with totals
	foreach (@querystrings) {
		($queryfield, $queryvalue) = split (/=/, $_);
		$queryvalue = uri_unescape($queryvalue);
		#print "FIELD $queryfield VALUE $queryvalue \n";
		$querytotals{$queryfield}++; # a count that each query field name has been seen
		$queryfields1{$queryfield}{$queryvalue}++; # key line. for this query field, log the value, and add a count to the value.
	}
	
	#print "URL " . $geturl . " and host $fields[0] and line $linecount\n"; 
	$clienttotal{$fields[0]}++;
	$browsertotal{$fields[9]}++;
		


}

print "Totals for sabre\n";
print "total hits used $hitcount and $skipcount records skipped \n";
print "Totals for computers used by users\n";
foreach $key (sort { $clienttotal{$b} <=> $clienttotal{$a} } keys %clienttotal) {
        printf "Client %4d %s\n", $clienttotal{$key}, $key;
    }
print "Totals for different browsers used\n";
foreach $key (sort { $browsertotal{$b} <=> $browsertotal{$a} } keys %browsertotal) {
        printf "Browser %4d %s\n", $browsertotal{$key}, $key;
    }
print "Totals for Pages\n";
foreach $key (sort { $filetotals{$b} <=> $filetotals{$a} } keys %filetotals) {
        printf "Page %4d %s\n", $filetotals{$key}, $key;
    }

print "Totals for the different fields used for queries\n";
foreach $key (sort { $querytotals{$b} <=> $querytotals{$a} } keys %querytotals) {
        printf "%4d hits included the %s field. \n", $querytotals{$key}, $key;
    }


while ( ($qfields2, $qvalues2) = each %queryfields1 ) {
   print "Query Field : $qfields2: \n";
   foreach $key (sort { $$qvalues2 {$b} <=> $$qvalues2 {$a}} keys %$qvalues2 ) {
	print "$qfields2 : $key \t \t $$qvalues2{$key} \n";
   }
print "\n";
}
